﻿using System.Collections;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;


namespace CrawlerConsole
{
    public class Crawler
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;

        public void Crawl()
        {
            Console.WriteLine("开始爬行。。。");
            while (true)
            {
                string? current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]!)
                    {
                        continue;
                    }
                    current = url;
                }
                if (current == null || count > 15)
                    break;
                Console.WriteLine("爬行" + current + "界面");
                string html = DownLoad(current);
                urls[current] = true;

                count++;
                Parse(html, current);
            }

        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient client = new WebClient();
                client.Encoding = Encoding.UTF8;
                string html = client.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;


            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }
        public void Parse(string html, string oldUrl)
        {
            //只有当爬取的是HTML文本时，才解析并爬取下一级URL
            string strRef = @"(href|HREF)[ ]*=[ ]*[""'](http|https)[^""'#>]+..html.*?[""']";

            MatchCollection matchCollection = new Regex(strRef).Matches(html);
            foreach (Match match in matchCollection)
            {
                var url = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (url.Length == 0)
                    continue;

                //相对地址转成绝对地址进行爬取
                Uri baseUri = new Uri(oldUrl);
                Uri absoluteUri = new Uri(baseUri, url);

                //只爬取起始网站上的网页
                if (url.Contains("https://www.cnblogs.com"))
                {
                    if (urls[absoluteUri.ToString()] == null)
                        urls[absoluteUri.ToString()] = false;
                }
            }
        }

    }
    public class Program
    {
        static void Main(string[] args)
        {
            Crawler crawler = new Crawler();
            string startUrl = "https://www.cnblogs.com/dstang2000/";
            if (args.Length > 0)
                startUrl = args[0];
            crawler.urls.Add(startUrl, false);
            new Thread(crawler.Crawl).Start();

        }
    }
}
